#!/usr/bin/env bash
#
# Copyright 2016 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Set PROGram name
PROG=${0##*/}
########################################################################
#+
#+ NAME
#+     $PROG - Kubernetes Release Tool
#+
#+ SYNOPSIS
#+     $PROG  <branch> [--yes] [--nomock] [--rc] [--official]
#+            [--clean] [--stage] [--github-release-draft]
#+            [--buildversion=<jenkins build version>]
#+            [--gcrio_path=<full gcr.io path>]
#+            [--basedir=<alt base work dir>]
#+            [--security_layer=/path/to/pointer/to/script]
#+            [--exclude-suites="<suite> ..."]
#+            [--build-at-head]
#+            [--prebuild] [--buildonly]
#+            [--mailto=<email1>,<email2>] [--gcb]
#+            [--tmpdir=</alt/tmp>]
#+     $PROG  [--helpshort|--usage|-?]
#+     $PROG  [--help|-man]
#+
#+ DESCRIPTION
#+     $PROG produces Kubernetes releases.
#+
#+     Driven by the named source <branch> and an optional [--rc] or
#+     [--official] flag, $PROG determines what needs to be released
#+     and calculates the version.
#+
#+     [--buildversion=] will override the automatic check for a build version
#+     Mostly used for testing as a way to re-use a known good build version.
#+
#+     What is going to be done is presented to the user and asks for
#+     confirmation before continuing.
#+
#+     All prerequisites are checked before the process begins.
#+
#+     $PROG runs in mock/dryrun mode by default.  To actually execute a
#+     release, pass in the --nomock flag.
#+
#+     ++++ NEW (for 1.8+) ++++
#+     --stage writes all usual and extra artifacts to
#+     gs://$RELEASE_BUCKET/stage for later use by 'release' runs.
#+     * The 'staged' version identifier is the source JENKINS_BUILD_VERSION
#+       rather than the computed version(s) that stem from it to identify
#+       the final release artifacts.  You may stage multiple source builds
#+       for potential release later
#+     * When you run a standard 'release', $PROG will look for staged bits:
#+       - LOCALLY: The complete source tree, including all commits & artifacts.
#+       - GCS: Artifacts only in
#+              gs://$RELEASE_BUCKET/stage/$JENKINS_BUILD_VERSION
#+     * This method works with both mock and --nomock INTERCHANGEABLY
#+       on the local disk. If relying on the GCS staging, be sure to use
#+       --nomock for both --stage and the real release run.
#+     * --stage also skips many of the usual release steps including many
#+       auth checks and pushes as it's primary function is to build and
#+       stage release bits and artifacts for later use.
#+
#+     Simply specify the <branch> you want to release from and follow the
#+     prompts.  <branch> is one of master, release-1.2, etc. and the release
#+     is based on the <branch> value and the [--official] flag:
#+
#+     WARNING: --build-at-head bypasses any test analysis and builds directly
#+     at the HEAD Of the named branch.  Mostly for testing.
#+
#+     Branch     RC  Official    Type
#+     ------     --  --------    ----
#+     master                     alpha
#+     master                     alpha
#+     master     X               N/A
#+     master            X        N/A
#+     release-*                  beta
#+     release-*  X               release candidate
#+     release-*         X        official
#+
#+     NOTE: <branch> can exist already or not.  If the branch doesn't exist,
#+           it will be branched from master as part of the run.
#+
#+     VALUES USED AND DISPLAYED DURING RUNS:
#+     * The RELEASE_VERSION dictionary is indexed by each of the types of
#+       releases that will be processed for a session (alpha,beta,rc,official)
#+     * RELEASE_VERSION_PRIME is the primary release version
#+     * GCRIO_PATH and RELEASE_BUCKET are the publish locations
#+     * Default GCRIO_PATH for non-mock runs utilizes a multi-region alias,
#+       k8s.gcr.io, and a single write alias, staging-k8s.gcr.io.
#+       * Switching between k8s.gcr.io and staging-k8s.gcr.io for the purposes
#+         of naming and pushing is handled by the tooling.  If --gcrio_path
#+         overrides, only that name is used by the tooling in all cases.
#+
#+     A simple $USER check controls who can run in --nomock mode as ACLs
#+     restrict who can actually push bits anyway.
#+
#+ OPTIONS
#+     [--stage]                 - Write all artifacts to
#+                                 gs://$RELEASE_BUCKET/stage for later use
#+     [--yes]                   - Assume 'yes' to all queries
#+     [--nomock]                - Complete an actual release with upstream
#+                                 pushes
#+     [--clean]                 - Force clean of previous session
#+     [--rc]                    - Release candidates on release branches only
#+     [--official]              - Official releases on release branches only
#+     [--buildversion=ver]      - Override Jenkins check and set a specific
#+                                 build version
#+     [--gcrio_path=]           - Specify the full GCR path to use.
#+                                 defaults:
#+                                 - gcr.io/kubernetes-release-test (mock)
#+                                 - k8s.gcr.io for --nomock
#+     [--basedir=dir]           - Specify an alternate base directory
#+                                 (default: $HOME/anago)
#+     [--security_layer=]       - A file containing a path to a script to
#+                                 source/include:
#+                                 FLAGS_security_layer=/path/to/script
#+                                 Default: $HOME/.kubernetes-releaserc
#+     [--exclude-suites=]       - Space separated list of CI suites regex to
#+                                 exclude from go/nogo criteria
#+     [--build-at-head]         - Force a build at HEAD of named branch
#+     [--mailto=]               - Comma-separated list of addresses to send
#+                                 announcement email to. Overrides default list.
#+     [--github-release-draft]  - Force a github release draft for mock runs.
#+                                 The draft placeholder if left behind can
#+                                 cause issues with the real release.
#+                                 Users are prompted to remove draft releases,
#+                                 but the action to create them in the first
#+                                 place should be explicit.
#+     [--gcb]                   - Running from GCB
#+     [--tmpdir=]               - Set an alternate temp dir
#+     [--prebuild]              - Used during GCB to halt before building
#+                                 to allow switch to build container
#+     [--buildonly]             - Used during GCB to halt after building
#+                                 to allow switch to release container
#+     [--help | -man]           - display man page for this script
#+     [--usage | -?]            - display in-line usage
#+
#+ EXAMPLES
#+     $PROG --yes master        - Do a mock alpha release from master
#+                                 and don't stop/prompt
#+     $PROG release-1.1         - Do a mock beta release from release-1.1
#+     $PROG --rc release-1.1
#+                               - Do a mock release candidate from release-1.1
#+     $PROG --official release-1.1
#+                               - Do a mock official release from release-1.1
#+     $PROG --nomock --official release-1.1
#+                               - Do an official release from release-1.1
#+     $PROG release-1.8 --stage --build-at-head
#+                               - Stage a HEAD build locally and on GCS
#+     $PROG release-1.8 --nomock --build-at-head
#+                               - Release the staged build
#+                                 (assuming HEAD hasn't changed)
#+
#+ FILES
#+     build/release.sh
#+
#+ SEE ALSO
#+     common.sh                 - common function definitions
#+     gitlib.sh                 - git/jenkins function definitions
#+     releaselib.sh             - release/push-specific functions
#+
#+ BUGS/TODO
#+
########################################################################
# If NO ARGUMENTS should return usage, uncomment the following line:
usage=${1:-yes}

# Deal with OSX limitations out the gate for anyone that tries this there
BASE_ROOT=$(dirname $(readlink -e "$BASH_SOURCE" 2>&1)) \
 || BASE_ROOT="$BASH_SOURCE"
source $BASE_ROOT/lib/common.sh
source $TOOL_LIB_PATH/gitlib.sh
source $TOOL_LIB_PATH/releaselib.sh

# Clear or validate run state
if ((FLAGS_clean)) && [[ -f $PROGSTATE ]]; then
  logrun mv $PROGSTATE $PROGSTATE.last
fi
common::validate_command_line "$ORIG_CMDLINE" || common::exit 1 "Exiting..."

# Validate command-line
common::argc_validate 1 || common::exit 1 "Exiting..."

# Set positional args
RELEASE_BRANCH=${POSITIONAL_ARGV[0]}

# Check branch format
[[ $RELEASE_BRANCH =~ $BRANCH_REGEX ]] \
 || common::exit 1 "Invalid branch name!"

# Check arg conflicts
if ((FLAGS_rc)) || ((FLAGS_official)); then
  if [[ "$RELEASE_BRANCH" == "master" ]]; then
    common::exit 1 "Can't do release candidate or official releases on master!"
  fi
fi

if ((FLAGS_rc)) && ((FLAGS_official)); then
  common::exit 1 "Can't do release candidate and official release!"
fi

###############################################################################
# FUNCTIONS
###############################################################################
###############################################################################
# common::cleanexit prog-specific override function
# Do stuff here to clean up after this specific script
# @param exit code
#
common::cleanexit () {
  [[ -t 1 ]] && tput cnorm

  logrun rm -rf $LOCAL_CACHE

  # Reset gcloud auth context
  # if GCP_USER is set then we are in a state to reset this
  if ! ((FLAGS_gcb)) && [[ -n "$GCP_USER" ]]; then
    logrun $GCLOUD config set account $GCP_USER
  fi

  if [[ -d $WORKDIR ]]; then
    if ((FLAGS_gcb)); then
      # Just copy the logs and files up. A completed release copies everything
      archive_release --files-only
    else
      copy_logs_to_workdir
    fi
  fi

  common::timestamp end
  exit ${1:-0}
}

###############################################################################
# Copy logs to WORKDIR
copy_logs_to_workdir () {
  local f
  local logfiles=$(ls $LOGFILE{,.[0-9]} 2>/dev/null || true)

  for f in $logfiles; do
    common::strip_control_characters $f
    common::sanitize_log $f
  done

  logecho -n "Copying $LOGFILE{,.[0-9]} to $WORKDIR: "
  logrun -s cp -f $logfiles $WORKDIR
}

###############################################################################
# Ensures all registries that will be used during both mock and --nomock
# runs allow write access so we don't fall over later
# @param registries - A space separated list of registries
#
ensure_registry_acls () {
  local registries=($1)
  local emptyfile="$TMPDIR/empty-file.$$"
  local gs_path
  local r
  local retcode=0
  local artifact_namespace

  # Make an empty file to test uploading
  logrun touch $emptyfile

  # Short of creating a hardcoded map of project-id to registry, translating
  # _ to - seems to be a simple rule to keep this, well, simple.
  for r in ${registries[*]//_/-}; do
    # In this context, "google-containers" is still used
    if [[ "$r" == "$GCRIO_PATH_PROD" ]]; then
      artifact_namespace="google-containers"
    else
      artifact_namespace="${r/gcr.io\//}"
    fi

    gs_path="gs://artifacts.$artifact_namespace.appspot.com/containers"
    logecho -n "Checking write access to registry $r: "
    if logrun $GSUTIL -q cp $emptyfile $gs_path && \
       logrun $GSUTIL -q rm $gs_path/${emptyfile##*/}; then
      logecho $OK
    else
      logecho $FAILED
      ((retcode++))
    fi

    # Always reset back to $USER
    ((FLAGS_gcb)) || logrun $GCLOUD config set account $GCP_USER
  done

  logrun rm -f $emptyfile

  return $retcode
}

###############################################################################
# Checks GCP users to ensure they are logged in and accessible
# return 1 on failure
ensure_gcp_users () {
  local user
  local -a users_to_check

  users_to_check+=($GCP_USER)

  for user in ${users_to_check[*]}; do
    logecho -n "Checking cloud account/auth $user: "
    if (logrun $GCLOUD config set account $user && \
        logrun $GCLOUD docker -- version >/dev/null 2>&1); then
      logecho -r "$OK"
    else
      logecho -r "$FAILED"
      logecho
      logecho "$user is not in the credentialed accounts list!"
      logecho "Sign in with:"
      logecho
      logecho "$ $GCLOUD auth login $user"
      return 1
    fi
  done
}

###############################################################################
# Checks release-specific prereqs
# Sets global GCLOUD_PROJECT
# @param package - A space separated list of packages to verify exist
# return 1 on failure
#
PROGSTEP[check_prerequisites]="CHECK PREREQUISITES"
check_prerequisites () {
  local rb

  # We check for docker here as a word which passes for docker-ce and
  # docker-engine as docker packages are in transiation of deprecating
  # docker-engine
  common::check_packages jq pandoc docker ${PREREQUISITE_PACKAGES[*]} \
   || return 1
  common::check_pip_packages yq || return 1

  security_layer::auth_check 2 || return 1

  logecho -n "Checking Docker version: "
  docker_version=$(docker version --format '{{.Client.Version}}' | cut -d"-" -f1)
  if [[ ${docker_version} != 18.06.0 && ${docker_version} < 18.06.0 ]]; then
    logecho "Minimum docker version 18.06.0 is required for " \
            "creating and pushing manifest images[found: ${docker_version}]"
    return 1
  fi
  logecho -r "$OK"

  # Ensure that the docker command line supports the manifest images
  export DOCKER_CLI_EXPERIMENTAL=enabled

  # TODO: Remove this section once docker manifest command promoted
  # from Experimental
  logecho -n "Verifying Docker CLI Experimental status: "
  cli_experimental=$(docker version --format '{{.Client.Experimental}}' | cut -d"-" -f1)
  if [[ "${cli_experimental}" == "false" ]]; then
    logecho "Docker Client Experimental flag is false, should be enabled to " \
            "push the manifest images"
    logecho "More info: https://docs.docker.com/edge/engine/reference/commandline/manifest_create/"
    return 1
  fi
  logecho -r "$OK"

  if ! ((FLAGS_gcb)); then
    ensure_gcp_users || return 1
  fi

  # Verify write access to all container registries that might be used
  # to ensure both mock and --nomock runs will work.
  ensure_registry_acls "${ALL_CONTAINER_REGISTRIES[*]}" || return 1

  logecho -n "Checking cloud project state: "
  GCLOUD_PROJECT=$($GCLOUD config get-value project 2>/dev/null)
  if [[ -z "$GCLOUD_PROJECT" ]]; then
    logecho -r "$FAILED"
    logecho "No account authorized through gcloud.  Please fix with:"
    logecho
    logecho "$ gcloud config set project <project id>"
    return 1
  fi
  logecho -r "$OK"

  # Verify write access to $WRITE_RELEASE_BUCKETS
  for rb in ${WRITE_RELEASE_BUCKETS[*]}; do
    release::gcs::check_release_bucket $rb || return 1
  done
}

###############################################################################
# Updates openapi-spec version files
# Uses the RELEASE_VERSION global dict
# @param label - label index to RELEASE_VERSION
rev_openapi_versions () {
  local label=$1
  local -a swagger_files=("api/openapi-spec/swagger.json"
                          "federation/apis/openapi-spec/swagger.json")
  local f

  #########################################################################
  # NOTE: To avoid a significant time sink from running update-all.sh as
  #       well as avoiding running the master-only doc stub creator
  #       update-generated-docs.sh, we simply and quickly modify the files that
  #       are dependent on a $version_file change
  #       If/when the great update-all.sh revolution occurs and this runs in
  #       under a minute, we can probably run that instead.
  #########################################################################
  for f in ${swagger_files[*]}; do
    # Handle the recent move of federation out of k/k
    if [[ ! -f $f ]]; then
      logecho "Skipping $f..."
      continue
    fi
    # Strip any suffix off incoming RELEASE_VERSION[$label]
    sed -i -r 's,"version": "'${VER_REGEX[release]}'","version": "'${RELEASE_VERSION[$label]//-*}'",g' $f
    logrun git add $f
  done

  # Only commit if the files have changed.
  if [[ -n "$(git status -s)" ]]; then
    logecho -n "Committing openapi-spec versioned files: "
    logrun -s git commit -am \
                "Kubernetes version ${RELEASE_VERSION[$label]} openapi-spec file updates"
  fi
}

###############################################################################
# Update $CHANGELOG_FILE on master
PROGSTEP[generate_release_notes]="GENERATE RELEASE NOTES"
generate_release_notes () {
  local release_tars=$TREE_ROOT/_output-$RELEASE_VERSION_PRIME/release-tars
  local action="Update"

  logecho -n "Generating release notes: "
  logrun -s relnotes $RELEASE_VERSION_PRIME --release-tars=$release_tars \
                     --branch=${PARENT_BRANCH:-$RELEASE_BRANCH} --htmlize-md \
                     --markdown-file=$RELEASE_NOTES_MD \
                     --html-file=$RELEASE_NOTES_HTML \
                     --release-bucket=$RELEASE_BUCKET || return 1

  logecho -n "Checkout master branch to make changes: "
  logrun -s git checkout master || return 1

  # The fetch and rebase before editing $CHANGELOG_FILE
  # avoids merge conflicts if another release cut
  # completed while we were building this one.
  logecho -n "Fetch origin/master to get latest $CHANGELOG_FILE: "
  logrun -s git fetch origin master || return 1
  logecho -n "Rebase on origin/master before editing $CHANGELOG_FILE: "
  logrun -s git rebase origin/master || return 1

  if [[ ! -f $CHANGELOG_FILE ]]; then
    cat<<EOF > $CHANGELOG_FILE
<!-- BEGIN MUNGE: GENERATED_TOC -->

<!-- END MUNGE: GENERATED_TOC -->

<!-- NEW RELEASE NOTES ENTRY -->
EOF
    action="Add"
    logecho -n "$CHANGELOG_FILE not found.  Creating: "
    logrun -s git add $CHANGELOG_FILE
  fi

  logecho -n "Insert $RELEASE_VERSION_PRIME notes into $CHANGELOG_FILE: "
  # Pipe to logrun() vs using directly, because quoting.
  sed -i -e 's/<!-- NEW RELEASE NOTES ENTRY -->/&\n/' \
         -e "/<!-- NEW RELEASE NOTES ENTRY -->/r $RELEASE_NOTES_MD" \
   $CHANGELOG_FILE | logrun -s

  logecho -n "Update $CHANGELOG_FILE TOC: "
  logrun -s common::mdtoc $CHANGELOG_FILE || return 1

  logecho -n "Committing $CHANGELOG_FILE: "
  logrun -s git commit -am \
            "$action $CHANGELOG_FILE for $RELEASE_VERSION_PRIME." \
   || return 1

  # Sync $CHANGELOG_FILE to release-* branch and clear all others
  if [[ "$RELEASE_BRANCH" =~ release- ]]; then
    logecho -n "Checkout $RELEASE_BRANCH branch to make changes: "
    logrun -s git checkout $RELEASE_BRANCH || return 1
    logecho -n "Remove any previous CHANGELOG-*.md files: "
    logrun -s git rm -f CHANGELOG-*.md || return 1
    logecho -n "Copy master $CHANGELOG_FILE to $RELEASE_BRANCH branch: "
    logrun -s git checkout master -- $CHANGELOG_FILE || return 1
    logecho -n "Committing $CHANGELOG_FILE: "
    logrun -s git commit -am \
              "Add/Update $CHANGELOG_FILE for $RELEASE_VERSION_PRIME." \
     || return 1
  fi
}

##############################################################################
# Checkout a git object (branch/tag/commit hash)
# @param label - The label to process
checkout_object () {
  local label="$1"
  local version="${RELEASE_VERSION[$label]}"
  local commit
  local tree_object="$RELEASE_BRANCH"
  local branch_arg
  local branch_point

  # When building and packaging on GCB, we split up the build and packaging
  # process so the prepared tree contains all of the tags and file changes
  # for the 2 release versions being built.
  # In that case, if the tag already exists, simply check it out for build
  # purposes.
  if ((FLAGS_gcb)) && ! ((FLAGS_prebuild)) && \
     git rev-parse "$version" >/dev/null 2>&1; then
    logecho -n "Checking out $version: "
    logrun -s git checkout $version || return 1
    return 0
  fi

  # Only check and extract VER_REGEX[build] if JENKINS_BUILD_VERSION is set.
  # It will not be set when branching from a tag (ex. release-1.7.5)
  # We do want to capture the sha1 however any time it is available
  # for use below in setting branch_point and tree_object.  Those cases are
  # covered by the fact that branch_point defaults to BRANCH_POINT (set globally
  # earlier in the pipeline) and in the case where we're just doing a straight
  # alpha release from master.
  if [[ -n $JENKINS_BUILD_VERSION ]]; then
    if ! [[ $JENKINS_BUILD_VERSION =~ ${VER_REGEX[build]} ]]; then
      logecho "Unable to set checkout point for release!" \
              "Invalid JENKINS_BUILD_VERSION=$JENKINS_BUILD_VERSION"
      return 1
    fi
    # Get the sha1 from VER_REGEX[build]
    commit="${BASH_REMATCH[2]}"
  fi

  # NOTE: For new branches, files are still also updated on master (CHANGELOG).
  # Therefore, while the new branch may be based on a commit earlier than
  # HEAD, all master activity must occur AT HEAD.
  if [[ -n "$PARENT_BRANCH" ]]; then
    if [[ $RELEASE_VERSION_PRIME == $version ]]; then
      if [[ $tree_object =~ release- ]] && \
         ! git rev-parse $tree_object >/dev/null 2>&1; then
        # Only create/reset (-B) and set a branch_point if the *release-*
        # branch doesn't already exist locally
        branch_arg="-b"
        # Use BRANCH_POINT if set, otherwise, the hash from
        # JENKINS_BUILD_VERSION
        branch_point=${BRANCH_POINT:-$commit}
      fi
    else
      # if this is not the PRIMary version on the named RELEASE_BRANCH, use the
      # parent
      tree_object=$PARENT_BRANCH
    fi
  else
    [[ $label == "alpha" ]] && tree_object="$commit"
  fi

  # The above code determines what and how we should check out and where we
  # should branch off. This information is captured in the variables
  # `$branch_arg`, `$tree_object`, and `$branch_point`.
  #
  # Previously, when we didn't use the `$checkout_args` array to collect all
  # the arguments to `git checkout ...`, the above mentioned variables were
  # passed unquoted to the `logrun` function. This had some interesting
  # consequences:
  # - empty variables, because they were not quoted, did not make it into the
  #   function, they basically vanished
  # - (potential) whitespaces or any characters interpreted by the shell would
  #   have broken this function
  #
  # Now we make sure to
  # - quote all our arguments
  # - exlicitely ignore empty variables as arguments, by not adding them to
  #   the `$checkout_args` array
  #
  # This means, `$checkout_args` will look like and result into the following
  # `git checkout ...` calls:
  # - ( 'release-1.12' )                             => git checkout 'release-1.12'
  # - ( '-b' 'release-1.12' '<shaToBanchOffFrom>' )  => git checkout '-b' 'release-1.12' '<shaToBranchOffFrom>'
  local checkout_args=()
  [ -n "${branch_arg}" ] && checkout_args+=( "${branch_arg}" )
  [ -n "${tree_object}" ] && checkout_args+=( "${tree_object}" )
  [ -n "${branch_point}" ] && checkout_args+=( "${branch_point}" )

  logecho -n "Checking out $tree_object (${checkout_args[*]}): "
  logrun -s git checkout "${checkout_args[@]}" || return 1
}

##############################################################################
# Commit a tag with a comment to the current HEAD of a branch
# @param label - The label to process
# @param branch - The branch to process
git_tag () {
  local label=$1
  local branch=$2
  local commit_string
  local label_common="$label"

  # Ensure a common name for label in case we're using the special beta indexes
  [[ "$label" =~ ^beta ]] && label_common="beta"

  # Tagging
  commit_string="Kubernetes $label_common release ${RELEASE_VERSION[$label]}"
  logecho -n "Tagging $commit_string on $branch: "
  logrun -s git tag -a -m "$commit_string" "${RELEASE_VERSION[$label]}"
}

##############################################################################
# Tag/Build a local kube_cross image based on the version in scope
PROGSTEP[local_kube_cross]="TAG/BUILD LOCAL KUBE CROSS"
local_kube_cross () {
  local version
  local image
  local local_image
  local docker_image

  docker_image="${GCRIO_PATH_PROD}/kube-cross"
  version="$(cat "${TREE_ROOT}/build/build-image/cross/VERSION")"
  image="${docker_image}:${version}"
  local_image="${docker_image}:local"

  logecho -n "Checking if ${image} exists locally: "
  if logrun -s docker pull "$image"; then
    logecho -n "Tagging docker image ${image} as ${local_image}: "
    logrun -s docker tag "$image" "$local_image"
  else
    logecho -n "Building docker image ${image}: "
    logrun -s docker build --cache-from "$docker_image" -t "$image" \
                           -t "$local_image" "${TREE_ROOT}/build/build-image/cross/"
  fi
}

##############################################################################
# Prepare sources for building for a given label
# @param label - The label to process
PROGSTEP[prepare_tree]="PREPARE AND TAG TREE"
prepare_tree () {
  local label=$1
  local label_common="$label"
  local branch

  # Check for tag first
  if git rev-parse "${RELEASE_VERSION[$label]}" >/dev/null 2>&1; then
    logecho "The ${RELEASE_VERSION[$label]} tag already exists!"
    logecho "Possible reasons for this:"
    logecho "* --buildversion is old."
    logecho "* $WORKDIR is unclean"
    return 1
  fi

  checkout_object $label || return 1

  # Now set the branch we're on
  branch=$(gitlib::current_branch)

  # If this is a new branch, rev openapi-spec version files
  # Because this modifies files on master and there's a good chance the
  # master has moved ahead by now, the tag has to occur before this file
  # change or the later rebase in gitlib::push_master() will rewrite the
  # commit associated with the tag and orphan it.
  if [[ -n "$PARENT_BRANCH" && $label == alpha ]]; then
    git_tag $label $branch || return 1
    rev_openapi_versions $label || return 1
    return 0
  fi

  # rev openapi-spec version files on branch for beta tags
  case $label in
    beta*) rev_openapi_versions $label || return 1 ;;
  esac

  # COMMENTING OUT FOR NOW AS THE WAY FORWARD IS NOT CLEAR
  # generate docs on new branches (from master) only
  # If the entirety of this session is based on a branch from master
  # (PARENT_BRANCH), and this iteration of prepare_tree() is operating on
  # the NON-master branch itself, versionize the docs
  #if [[ "$PARENT_BRANCH" == "master" && "$branch" != "master" ]]; then
  #  logecho -n "Generating docs for ${RELEASE_VERSION[$label]}: "
  #  logrun -s $TREE_ROOT/hack/generate-docs.sh || return 1
  #  logecho -n "Committing: "
  #  logrun -s git commit -am \
  #            "Generating docs for ${RELEASE_VERSION[$label]} on $branch."
  #fi

  # Clear all CHANGELOG-N.NN.md files on the branch.
  # This replaces the above commented out change to the branch.
  # Some unique change is required on the branch to allow it to be git
  # describe'd as something other than the current tag on master
  # If/when the generate-docs.sh issue is sorted out, this will become
  # optional, though probably still useful anyway.
  if [[ "$PARENT_BRANCH" == "master" && "$branch" != "master" ]]; then
    logecho -n "Remove any previous CHANGELOG-*.md files: "
    logrun -s git rm -f CHANGELOG-*.md || return 1
    logecho -n "Copy master $CHANGELOG_FILE to $branch branch: "
    logrun -s git checkout master -- $CHANGELOG_FILE || return 1
    logecho -n "Committing deleted CHANGELOG-*.md files: "
    logrun -s git commit -am \
              "Delete extraneous CHANGELOG-*.md files on branch." \
     || return 1
  fi

  git_tag $label $branch
}

##############################################################################
# Make cross only
# This very low level split of the build required to run exactly 'make cross'
# only using the kube-cross container image
# @param label - The label to process
PROGSTEP[make_cross]="MAKE CROSS"
make_cross () {
  local label=$1
  local version=${RELEASE_VERSION[$label]}
  local branch

  checkout_object $label

  branch=$(gitlib::current_branch)

  # Convert a detached head state to something more readable
  [[ "$branch" == HEAD ]] && branch="master (detached head)"

  # For official releases we need to build BOTH the official and the beta and
  # push those releases.
  logecho -n "make cross only $version on $branch: "

  logrun -s -v make cross-in-a-container KUBE_DOCKER_IMAGE_TAG="$version" \
   || return 1

  logecho -n "Moving build _output to $BUILD_OUTPUT-$version: "
  logrun -s mv $BUILD_OUTPUT $BUILD_OUTPUT-$version || return 1
}

##############################################################################
# Build the Kubernetes tree
# @param label - The label to process
PROGSTEP[build_tree]="BUILD TREE"
build_tree () {
  local label=$1
  local version=${RELEASE_VERSION[$label]}
  local branch

  checkout_object $label

  branch=$(gitlib::current_branch)

  # Convert a detached head state to something more readable
  [[ "$branch" == HEAD ]] && branch="master (detached head)"

  # For official releases we need to build BOTH the official and the beta and
  # push those releases.
  logecho -n "Building Kubernetes $version on $branch: "

  if ((FLAGS_gcb)); then
    # make_cross done separately
    # OUT_DIR works in this context.  Woohoo!
    # KUBE_DOCKER_IMAGE_TAG needed here due to references deep in
    # build/lib/release.sh
    logrun -s -v make package-tarballs KUBE_DOCKER_IMAGE_TAG="$version" \
                                       OUT_DIR=$OUT_DIR-$version || return 1
  else
    # TODO: Ideally we update LOCAL_OUTPUT_ROOT in build/common.sh to be
    #       modifiable.  In the meantime just mv the dir after it's done
    # Not until https://github.com/kubernetes/kubernetes/issues/23839
    #logrun -s make release OUT_DIR=$BUILD_OUTPUT-${RELEASE_VERSION[$label]}
    logrun -s make release KUBE_DOCKER_IMAGE_TAG="$version" || return 1

    logecho -n "Moving build _output to $BUILD_OUTPUT-$version: "
    logrun -s mv $BUILD_OUTPUT $BUILD_OUTPUT-$version || return 1
  fi
}

##############################################################################
# Look for binary artifacts in the known loctions
# @param base - The base directory
# @param version - Version
# return 1 on failure
binary_artifacts_exist () {
  local base=$1
  local version=$2
  local gsutil

  # Detect GSC paths
  [[ $base =~ ^gs:// ]] && gsutil="$GSUTIL -q"

  logecho -n "Searching for artifacts at $base: "
  if logrun $gsutil ls $base/gcs-stage/$version/*.tar.gz* >/dev/null 2>&1 && \
     logrun $gsutil ls $base/release-images/*/*.tar >/dev/null 2>&1; then
    logecho "$FOUND"
  else
    logecho "$NOTFOUND"
    return 1
  fi
}

##############################################################################
# Can be one of
# * Local disk src - symlink it
# * src.tar.gz, gcs and gcr artifacts on GCS
# Sets global STAGED_LOCATION={local,gcs}
# Sets global STAGED_BUCKET
# Modifies global RELEASE_GB
# returns 1 on failure
found_staged_location () {
  local release_versions="$*"
  local bucket
  local jbv="$JENKINS_BUILD_VERSION"
  local local_root="$BASEDIR/$PROG-$jbv/src"
  local k8s_root="$local_root/k8s.io/kubernetes/_output"
  local gs_stage_root
  local tmp_tar_archive=$TMPDIR/$PROG-$$-src.tar.gz

  logecho
  # first look locally
  for version in $release_versions; do
    if binary_artifacts_exist $k8s_root-$version $version; then
      STAGED_LOCATION="local"
    else
      STAGED_LOCATION=""
      break
    fi
  done

  # NOTE: We could also look for artifacts on GCS even when found local
  # in order to do a bucket-to-bucket copy and save 2 minutes.
  # But this isn't a typical use-case.  Normally staging will happen
  # exclusively on GCB and then 'release' will happen either on the desktop
  # or GCB which will never find anything 'local' anyway.
  if [[ $STAGED_LOCATION == "local" ]]; then
    # Enjoy the local filesystem
    logecho -n "Symlinking $local_root to $WORKDIR/src: "
    logrun mkdir -p $WORKDIR
    logrun -s ln -nsf $local_root $WORKDIR/src || return 1
    RELEASE_GB="5"
    return 0
  fi

  # If that didn't work out, look for source archive
  # and artifacts on all READ_RELEASE_BUCKETS on GCS
  for bucket in ${READ_RELEASE_BUCKETS[*]}; do
    logecho -n "Searching for staged src.tar.gz on $bucket: "

    if logrun $GSUTIL -m cp gs://$bucket/stage/$jbv/src.tar.gz \
                         $tmp_tar_archive; then
      logecho "$FOUND"
      # Set a global to skip other build steps in workflow
      STAGED_LOCATION="gcs"
      STAGED_BUCKET="$bucket"
      break
    else
      logecho "$NOTFOUND"
    fi
  done

  # If we get here and STAGED_LOCATION isn't set, give up
  [[ -z "$STAGED_LOCATION" ]] && return 1

  # Now ensure binary artifacts exist in the same location, or give up
  for version in $release_versions; do
    gs_stage_root="gs://$bucket/stage/$jbv/$version"
    if ! binary_artifacts_exist $gs_stage_root $version; then
      # Ensure these are unset in this case
      unset STAGED_LOCATION STAGED_BUCKET
      return 1
    fi
  done

  logecho -n "Extracting $tmp_tar_archive: "
  logrun -s tar xfz $tmp_tar_archive -C $WORKDIR || return 1
  logrun rm -f $tmp_tar_archive

  # Set disk requirements when everything's on GCS
  RELEASE_GB="10"
}

##############################################################################
# Copy artifacts from GCS and between buckets as needed
# @param label - The ${RELEASE_VERSION{index}]
# @param staged_bucket - The STAGED_BUCKET
# @param release_bucket - The RELEASE_BUCKET
# return 1 on failure
copy_staged_from_gcs () {
  local label=$1
  local staged_bucket=$2
  local release_bucket=$3
  local jbv="$JENKINS_BUILD_VERSION"
  local version="${RELEASE_VERSION[$label]}"
  local gs_stage_root="gs://$staged_bucket/stage/$jbv/$version"
  local gs_release_root="gs://$release_bucket/release/$version"
  local outdir="$TREE_ROOT/_output-$version"
  local type

  # cp the /stage/ tarballs to /release/ on GCS directly
  logecho -n "Bucket-to-bucket copy $gs_stage_root/gcs-stage artifacts" \
             "to $gs_release_root: "
  logrun -s $GSUTIL -mq cp -rc $gs_stage_root/gcs-stage/$version/* \
                               $gs_release_root/ || return 1

  logrun mkdir -p $outdir/gcs-stage/$version
  logecho -n "Copy staged kubernetes.tar.gz to $outdir/gcs-stage/$version: "
  logrun -s $GSUTIL -q \
            cp -c $gs_stage_root/gcs-stage/$version/kubernetes.tar.gz \
                  $outdir/gcs-stage/$version || return 1

  # Copy docker images back to _output trees for later pushing
  # TODO: Can we somehow stage these on GCR.IO using docker or even
  #       the GCS backend to eliminate the VERY EXPENSIVE --12 minutes--
  #       it takes to push these containers from local disk?
  logrun mkdir -p $outdir/release-images
  logecho -n "Copy staged docker images to $outdir/release-images: "
  logrun -s $GSUTIL -mq cp -cr $gs_stage_root/release-images/* \
                               $outdir/release-images/ || return 1
}

##############################################################################
# Push git objects to github
# NOTES:
# * alpha is alone, pushes tags only
# * beta is alone, pushes branch and tags
# * rc is alone, pushes branch and tags
# * official pushes both official and beta items - branch and tags
# * New branch tags a new alpha on master, new beta on new branch and pushes
#   new branch and tags on both
PROGSTEP[push_git_objects]="PUSH GIT OBJECTS"
push_git_objects () {
  local b
  local dryrun_flag=" --dry-run"

  # The real deal?
  ((FLAGS_nomock)) && dryrun_flag=""

  ((FLAGS_yes)) \
   || common::askyorn -e "Pausing here. Confirm push$dryrun_flag of tags" \
                         "and bits" \
   || common::exit 1 "Exiting..."

  logecho -n "Checkout master branch to push objects: "
  logrun -s git checkout master || return 1

  logecho "Pushing$dryrun_flag tags"
  for b in ${!RELEASE_VERSION[@]}; do
    logecho -n "* ${RELEASE_VERSION[$b]}: "
    logrun -s git push$dryrun_flag origin ${RELEASE_VERSION[$b]} || return 1
  done

  if [[ "$RELEASE_BRANCH" =~ release- ]]; then
    logecho -n "Pushing$dryrun_flag $RELEASE_BRANCH branch: "
    logrun -s git push$dryrun_flag origin $RELEASE_BRANCH || return 1
    # Additionally push the parent branch if a branch of branch
    if [[ "$PARENT_BRANCH" =~ release- ]]; then
      logecho -n "Pushing$dryrun_flag $PARENT_BRANCH branch: "
      logrun -s git push$dryrun_flag origin $PARENT_BRANCH || return 1
    fi
  fi

  # For files created on master with new branches and
  # for $CHANGELOG_FILE, update the master
  gitlib::push_master
}

###############################################################################
# generate the announcement text to be mailed and published
branch_announcement_text () {
  cat <<EOF
Kubernetes team,
<P>
Kubernetes' $RELEASE_BRANCH branch has been created.
<P>
The release owner will be sending updates on how to interact with this branch shortly.  The <A HREF=https://git.k8s.io/community/contributors/devel/sig-release/cherry-picks.md>Cherrypick Guide</A> has some general guidance on how things will proceed.
<P>
Announced by <A HREF=https://github.com/kubernetes/release>$PROG</A>, the Kubernetes Release Tool
EOF
}

###############################################################################
# generate the announcement text to be mailed and published
release_announcement_text () {
  cat <<EOF
Kubernetes team,
<P>
Kubernetes $RELEASE_VERSION_PRIME has been built and pushed.
<P>
The release notes have been updated in <A HREF=https://github.com/kubernetes/kubernetes/blob/master/$CHANGELOG_FILE/#${RELEASE_VERSION_PRIME//\./}>$CHANGELOG_FILE</A> with a pointer to it on <A HREF=https://github.com/kubernetes/kubernetes/releases/tag/$RELEASE_VERSION_PRIME>github</A>:
<P>
<HR>
$(cat $RELEASE_NOTES_HTML)
<HR>
<P><BR>
Leads, the <A HREF=https://github.com/kubernetes/kubernetes/blob/master/$CHANGELOG_FILE/#${RELEASE_VERSION_PRIME//\./}>$CHANGELOG_FILE</A> has been bootstrapped with $RELEASE_VERSION_PRIME release notes and you may edit now as needed.
<P><BR>
Published by <A HREF=https://github.com/kubernetes/release>$PROG</A>, the Kubernetes Release Tool
EOF
}

###############################################################################
# Construct META for the announcement and send it out
# @optparam --branch - Default announcement type is 'release' or --branch.
PROGSTEP[announce]="ANNOUNCE BRANCH OR RELEASE"
announce () {
  local arg="$1"
  local announcement_file="${WORKDIR}/announcement.html"
  local subject_file="${WORKDIR}/announcement-subject.txt"
  local nomock_flag=""
  local pubotRc=0
  local pubotIssue=''

  logecho "Creating k8s ${RELEASE_VERSION_PRIME} announcement in ${WORKDIR} ..."

  if [[ "$arg" == "--branch" ]]; then
    echo "k8s ${RELEASE_BRANCH} branch has been created" > "$subject_file"
    branch_announcement_text > "$announcement_file"

    # When we create a new branch, we notify the publishing-bot folks by
    # creating an issue for them
    pubotIssue="$( gitlib::create_publishing_bot_issue "$RELEASE_BRANCH" | gitlib::get_issue_url )" || pubotRc=$?
    if [ "$pubotRc" != '0' ]; then
      logecho "${WARNING}: Could not create issue for the publishing-bot update"
    else
      logecho "${OK}: publishing-bot update issue created: ${pubotIssue}"
    fi
  else
    echo "k8s ${RELEASE_VERSION_PRIME} is live!" > "$subject_file"
    release_announcement_text > "$announcement_file"
  fi

  # Only send from desktop
  if ((FLAGS_gcb)); then
    logecho "$WARNING: Email cannot be sent from GCB.  Execute the following" \
            "after a completed release to send email notification:"
    logecho

    # for an explanation of this construct look at
    #
    # http://tldp.org/LDP/abs/html/testconstructs.html
    #
    # or for the tl;dr check out
    #
    # https://www.tldp.org/LDP/abs/html/dblparens.html
    #
    # what we're doing here is setting nomock_flag="--nomock"
    # if FLAGS_nomock evaluates to true...it's concise and consistent
    # with the rest of anago but replicate the pattern at your peril
    # definitely understand what's happening when you do
    #
    # myVar="OG"; ((0)) && myVar="changed"; echo $myVar
    #
    # vs
    #
    # myVar="OG"; ((1)) && myVar="changed"; echo $myVar
    # note also that the values for FLAGS_x are either 0 (not set)
    # or 1 (set) so that this works. This is not Ruby or similar
    # where "--some-flag" would be truthy but it is BASH
    # so values > 0 are actually truthy
    ((FLAGS_nomock)) && nomock_flag="--nomock"

    logecho "$ release-notify ${nomock_flag} ${RELEASE_VERSION_PRIME}"
    logecho
  else
    release::send_announcement || return 1
  fi
}

###############################################################################
# Update the releases page on github
# return 1 on failure
PROGSTEP[update_github_release]="UPDATE GITHUB RELEASES PAGE"
update_github_release () {
  local release_id
  local id_suffix
  local release_verb="Posting"
  local prerelease="true"
  local draft="true"
  local staging_dir="${TREE_ROOT}/_output-${RELEASE_VERSION_PRIME}/gcs-stage/${RELEASE_VERSION_PRIME}"
  local tarball="${staging_dir}/kubernetes.tar.gz"
  local sha256_hash
  local sha512_hash
  local sha256sums_file="${staging_dir}/SHA256SUMS"
  local sha512sums_file="${staging_dir}/SHA512SUMS"

  sha256_hash=$(common::sha "$tarball" 256)
  sha512_hash=$(common::sha "$tarball" 512)

  ((FLAGS_official)) && prerelease="false"
  if ((FLAGS_nomock)); then
    draft="false"

    # Check to see that a tag exists.
    # non-draft release posts to github create a tag.  We don't want to
    # create any tags on the repo this way.  The tag should already exist
    # as a result of the release process.
    if ! $GHCURL $K8S_GITHUB_API/git/refs/tags |jq -r '.[] | .ref' |\
        egrep -q "^refs/tags/$RELEASE_VERSION_PRIME$"; then
      logecho
      logecho "$FATAL: How did we get here?"
      logecho "The $RELEASE_VERSION_PRIME tag doesn't exist yet on github." \
              "That can't be good."
      logecho "We certainly cannot publish a release without a tag."
      return 1
    fi
  else
    # Return unless explicitly requested
    if ! ((FLAGS_github_release_draft)); then
      logecho "Mock run - Skipping.  Use --github-release-draft to force."
      return
    fi
  fi

  # Does the release exist yet?
  release_id=$($GHCURL $K8S_GITHUB_API/releases/tags/$RELEASE_VERSION_PRIME |\
               jq -r '.id')

  if [[ -n "$release_id" ]]; then
    logecho "The $RELEASE_VERSION_PRIME is already published on github."
    if ((FLAGS_yes)) || common::askyorn -e "Would you like to update it"; then
      logecho "Setting post data id to $release_id to update existing release."
      id_suffix="/$release_id"
      release_verb="Updating"
    else
      logecho "Existing release (id #$release_id) left intact."
      return 1
    fi
  fi

  # post release data
  logecho "$release_verb the $RELEASE_VERSION_PRIME release on github..."
  local changelog_url="$K8S_GITHUB_URL/blob/master/$CHANGELOG_FILE"
  release_id=$($GHCURL $K8S_GITHUB_API/releases$id_suffix --data \
   '{
    "tag_name": "'$RELEASE_VERSION_PRIME'",
    "target_commitish": "'$RELEASE_BRANCH'",
    "name": "'$RELEASE_VERSION_PRIME'",
    "body": "See [kubernetes-announce@](https://groups.google.com/forum/#!forum/kubernetes-announce) and ['$CHANGELOG_FILE']('$changelog_url'#'${RELEASE_VERSION_PRIME//\./}') for details.\n\nSHA256 for `kubernetes.tar.gz`: `'${sha256_hash}'`\n\nSHA512 for `kubernetes.tar.gz`: `'${sha512_hash}'`\n\nAdditional binary downloads are linked in the ['$CHANGELOG_FILE']('$changelog_url'#downloads-for-'${RELEASE_VERSION_PRIME//\./}').",
    "draft": '$draft',
    "prerelease": '$prerelease'
    }' |jq -r '.id')

  # verify it was created
  if [[ -z "$release_id" ]]; then
    logecho
    logecho -r "$FAILED to create the $RELEASE_VERSION_PRIME release on github!"
    return 1
  fi

  local assets_upload_url="${K8S_GITHUB_API/api\./uploads\.}/releases/${release_id}/assets"

  # publish binary
  logecho -n "Uploading binary to github: "
  if $GHCURL -H "Content-Type:application/x-compressed" \
   --data-binary "@$tarball" \
   "${assets_upload_url}?name=${tarball##*/}"; then
    logecho "$OK"
  else
    logecho "$FAILED"
  fi

  local file

  # Upload SHA sums to GitHub
  logecho -n "Uploading SHA sums to github: "
  for file in "$sha256sums_file" "$sha512sums_file"; do
    if [[ ! -f $file ]]; then
      logecho -n "Unable to find $file. Skipping..."
    else
      if $GHCURL -H "Content-Type:application/octet-stream" \
      --data-binary @$file \
      "${assets_upload_url}?name=${file##*/}"; then
        logecho "$OK"
      else
        logecho "$FAILED"
      fi
    fi
  done

  if $draft; then
    logecho
    logecho "$ATTENTION: A draft release of $RELEASE_VERSION_PRIME was" \
            "created at $K8S_GITHUB_URL/releases."
    logecho

    # delete it
    if ((FLAGS_yes)) || \
       common::askyorn -y "Delete draft release (id #$release_id) now"; then
      logecho -n "Deleting the draft release (id #$release_id): "
      $GHCURL -X DELETE $K8S_GITHUB_API/releases/$release_id
    fi

    # verify it was deleted
    release_id=$($GHCURL $K8S_GITHUB_API/releases/$release_id | jq -r '.id')
    if [[ -n "$release_id" ]]; then
      logecho -r $FAILED
      logecho "The draft release (id #$release_id) was NOT deleted." \
              "Deal with it by hand"
      logecho "$K8S_GITHUB_URL/releases/$RELEASE_VERSION_PRIME"
    else
      logecho -r $OK
    fi
  fi
}

##############################################################################
# When creating new alphas on the master branch, make sure that X.Y-1.0
# has been created first or warn.
# This is to ensure that branchff can continue to be used as needed.  User
# can override here.
# @param buildver - Incoming JENKINS_BUILD_VERSION
branchff_sanity_check () {
  local buildver=$1
  local latest_official

  if [[ ! $buildver =~ ${VER_REGEX[release]} ]]; then
    logecho "Invalid format: $buildver"
    return 1
  fi

  # For master branch alpha builds, ensure we've released the previous .0
  # first
  if [[ -z "$PARENT_BRANCH" && -n ${RELEASE_VERSION[alpha]} ]]; then
    latest_official=${BASH_REMATCH[1]}.$((${BASH_REMATCH[2]}-1))
    # The $'\n` construct below is a word boundary.
    if [[ ! "$($GHCURL $K8S_GITHUB_API/tags |jq -r '.[] .name')" =~ \
       $'\n'v$latest_official.0$'\n' ]]; then
      logecho
      logecho "$WARNING:" \
              "$latest_official.0 hasn't been tagged/created yet." \
              "Creating ${RELEASE_VERSION[alpha]} *will* preclude any" \
              "future branch fast-forwards from master to" \
              "release-$latest_official."

      if ! ((FLAGS_yes)); then
        logecho "Are you *really* sure you want to do this?"
        common::askyorn "Continue creating ${RELEASE_VERSION[alpha]} now" \
         || return 1
      fi
    fi
  fi
}

##############################################################################
# Calls into Jenkins looking for a build to use for release
# Sets global PARENT_BRANCH when a new branch is created
# And global BRANCH_POINT when new branch is created from an existing tag
PROGSTEP[get_build_candidate]="SET BUILD CANDIDATE"
get_build_candidate () {
  local testing_branch
  local branch_head=$($GHCURL $K8S_GITHUB_API/commits/$RELEASE_BRANCH |\
                      jq -r '.sha')
  # Shorten
  branch_head=${branch_head:0:14}

  # Are we branching to a new branch?
  if gitlib::branch_exists $RELEASE_BRANCH; then
    # If the branch is a 3-part branch (ie. release-1.2.3)
    if [[ $RELEASE_BRANCH =~ $BRANCH_REGEX ]] && \
       [[ -n ${BASH_REMATCH[4]} ]]; then
      ((FLAGS_official)) \
       || common::exit 1 "--official required on 3-part branches!"

      # The 'missing' . here between 2 and 3 is intentional. It's part of the
      # optional regex.
      BRANCH_POINT=v${BASH_REMATCH[1]}.${BASH_REMATCH[2]}${BASH_REMATCH[3]}
    fi
    testing_branch=$RELEASE_BRANCH
  else
    [[ $RELEASE_BRANCH =~ $BRANCH_REGEX ]]

    # Not a 3-part branch
    if [[ -z "${BASH_REMATCH[4]}" ]]; then
      if ((FLAGS_official)); then
        common::exit 1 "Can't do official releases when creating a new branch!"
      fi

      PARENT_BRANCH=master
      testing_branch=$PARENT_BRANCH
    # if 3 part branch name, check parent exists
    elif gitlib::branch_exists ${RELEASE_BRANCH%.*}; then
      PARENT_BRANCH=${RELEASE_BRANCH%.*}
      # The 'missing' . here between 2 and 3 is intentional. It's part of the
      # optional regex.
      BRANCH_POINT=v${BASH_REMATCH[1]}.${BASH_REMATCH[2]}${BASH_REMATCH[3]}
      testing_branch=$PARENT_BRANCH
    else
      common::exit 1 "$FATAL! We should never get here! branch=$RELEASE_BRANCH"
    fi
  fi

  if [[ -z $BRANCH_POINT ]]; then
    if [[ -n "$FLAGS_buildversion" ]]; then
      logecho -r "$ATTENTION: Using --buildversion=$FLAGS_buildversion"
      JENKINS_BUILD_VERSION="$FLAGS_buildversion"
    else
      logecho "Asking Jenkins for a good build (this may take some time)..."
      FLAGS_verbose=1 release::set_build_version \
       $testing_branch "" "$FLAGS_exclude_suites" "100" || return 1
    fi

    # The RELEASE_BRANCH should always match with the JENKINS_BUILD_VERSION
    if [[ $RELEASE_BRANCH =~ release- ]] && \
       [[ ! $JENKINS_BUILD_VERSION =~ ^v${RELEASE_BRANCH/release-/} ]]; then
      logecho
      logecho "$FATAL!  branch/build mismatch!"
      logecho "buildversion=$JENKINS_BUILD_VERSION branch=$RELEASE_BRANCH"
      common::exit 1
    fi

    # Check state of master branch before continuing
    branchff_sanity_check $JENKINS_BUILD_VERSION
  else
    # The build version should never be behind HEAD on release existing branches
    if [[ "$RELEASE_BRANCH" =~ release-([0-9]{1,})\. ]]; then
      if [[ $JENKINS_BUILD_VERSION =~ ${VER_REGEX[build]} && \
            ${BASH_REMATCH[2]} != $branch_head ]]; then
        logecho
        logecho "$FATAL: The $RELEASE_BRANCH HEAD is ahead of the chosen" \
                "commit. Releases on release branches must be run from HEAD."
        return 1
      fi
    fi
  fi
}

##############################################################################
# Computes release values and sets the global RELEASE_VERSION[] dict and
# RELEASE_VERSION_PRIME (the primary release for this session).
# Also ensures the RELEASE_VERSION_PRIME tag doesn't already exist.
PROGSTEP[set_release_values]="SET RELEASE VALUES"
set_release_values () {
  FLAGS_verbose=1 \
   release::set_release_version ${BRANCH_POINT:-$JENKINS_BUILD_VERSION} \
                                $RELEASE_BRANCH \
                                $PARENT_BRANCH \
   || return 1

  # Check that this tag doesn't exist. Staged builds may be old
  if [[ "$($GHCURL $K8S_GITHUB_API/tags |jq -r '.[] .name')" =~ \
        $'\n'$RELEASE_VERSION_PRIME$'\n' ]]; then
     logecho
     logecho "$FATAL: The tag $RELEASE_VERSION_PRIME already exists on github."
     logecho "An old --buildversion was specified on the command-line."
     return 1
  fi

}

##############################################################################
# Prepare the workspace and sync the tree
PROGSTEP[prepare_workspace]="PREPARE WORKSPACE"
prepare_workspace () {
  local outdir

  if [[ -h $WORKDIR/src ]]; then
    # The case where an existing release tree might have a symlink from a
    # previously failed attempt.  We don't want to fall into the condition
    # below and clean it up
    logrun rm -f $WORKDIR/src || return 1
  else
    if [[ -d $TREE_ROOT ]]; then
      logecho "Checking for _output directories..."
      logrun cd $TREE_ROOT
      # set -e sillyness - yes that's a ||true there that would otherwise not
      # be needed except for 'set -e' in all its glory
      for outdir in $(ls -1d _output* 2>/dev/null ||true); do
        # This craziness due to
        # https://github.com/kubernetes/kubernetes/issues/23839
        if [[ $outdir != "_output" ]]; then
          logrun mv $outdir _output
        fi
        logecho -n "make clean for $outdir: "
        logrun -s make clean || return 1
        logecho -n "Removing _output: "
        logrun -s rm -rf _output || return 1
      done
    fi
    logecho -n "Removing/recreating $WORKDIR: "
    logrun cd $TMPDIR
    logrun -s rm -rf $WORKDIR || return 1
    logrun mkdir -p $WORKDIR || return 1
  fi

  # Check if locally staged directory contains all of the RELEASE_VERSIONs for
  # this release type
  if ((FLAGS_stage)) || ! found_staged_location ${RELEASE_VERSION[@]}; then
    # GCB releases (gcbmgr release) requires a found staged build to use
    if ! ((FLAGS_stage)) && ((FLAGS_gcb)); then
      logecho "$FATAL: Releasing from GCB requires a staged" \
              "--buildversion=<buildversion>"
      return 1
    fi
    # Sync the tree
    gitlib::sync_repo $K8S_GITHUB_AUTH_URL $TREE_ROOT || return 1
  fi
}


##############################################################################
# Archive the release on GS
# @param --files-only - non-recursive
archive_release () {
  local arg=$1
  local archive_bucket="gs://$RELEASE_BUCKET/archive"
  local build_dir=${WORKDIR##*/}
  local text="files"

  if ((FLAGS_yes)) || \
     common::askyorn -y "Archive this release on $archive_bucket"; then
    # Keep going
    :
  else
    return 0
  fi

  copy_logs_to_workdir

  # TODO: Copy $PROGSTATE as well to GCS and restore it if found
  # also delete if complete or just delete once copied back to $TMPDIR
  # This is so failures on GCB can be restarted / reentrant too.

  if [[ $arg != "--files-only" ]]; then
    dash_args="-rc"
    text="contents"
  fi

  # Remove temporary password file so not to alarm passers-by.
  logrun find $WORKDIR -type f -name rsyncd.password -delete ||true

  # Best effort
  logecho "Copy $WORKDIR $text to $archive_bucket/$build_dir..."
  logrun $GSUTIL -mq cp $dash_args $WORKDIR/* $archive_bucket/$build_dir || true

  logecho -n "Ensure PRIVATE ACL on" \
             "$archive_bucket/$build_dir/${LOGFILE##*/}\*: "
  logrun -s $GSUTIL acl ch -d AllUsers \
            "$archive_bucket/$build_dir/${LOGFILE##*/}*" || true
}

##############################################################################
# Stages the completed tar'd up source tree archive on GCS for later use
# returns 1 on failure
PROGSTEP[stage_source_tree]="STAGE SOURCE TREE"
stage_source_tree () {
  local jbv="$JENKINS_BUILD_VERSION"

  logecho -n "Tar up staged source tree: "
  logrun -s tar cvfz $WORKDIR/src.tar.gz -C $WORKDIR src --exclude="_output-*" \
   || return 1
  logecho -n "Archive fully staged source tree on GCS: "
  logrun -s $GSUTIL -m cp $WORKDIR/src.tar.gz \
            gs://$RELEASE_BUCKET/$BUCKET_TYPE/$jbv/src.tar.gz || return 1
  # Clean up
  logrun rm -f $WORKDIR/src.tar.gz || return 1
}

##############################################################################
# Pushes all binary and pointer artifacts up to GCS and GCR
# @param label - the label index for the version being pushed
# returns 1 on failure
PROGSTEP[push_all_artifacts]="PUSH BINARY RELEASE ARTIFACTS"
push_all_artifacts () {
  local label=$1
  local version="${RELEASE_VERSION[$label]}"
  local jbv="$JENKINS_BUILD_VERSION"

  if [[ -z "$STAGED_LOCATION" ]]; then
    # Locally Stage the release artifacts in build directory (gcs-stage)
    common::runstep release::gcs::locally_stage_release_artifacts \
     $BUCKET_TYPE $version $BUILD_OUTPUT-$version || return 1
  fi

  # The full release stage case
  if ((FLAGS_stage)); then
    # Push gcs-stage to GCS
    common::runstep release::gcs::push_release_artifacts \
     $BUILD_OUTPUT-$version/gcs-stage/$version \
     gs://$RELEASE_BUCKET/$BUCKET_TYPE/$jbv/$version/gcs-stage/$version \
      || return 1

    # Push docker release-images to GCS
    common::runstep release::gcs::push_release_artifacts \
     $BUILD_OUTPUT-${RELEASE_VERSION[$label]}/release-images \
     gs://$RELEASE_BUCKET/$BUCKET_TYPE/$jbv/$version/release-images || return 1
  else
    if [[ $STAGED_LOCATION == "gcs" ]]; then
      common::runstep copy_staged_from_gcs $label $STAGED_BUCKET \
                                           $RELEASE_BUCKET || return 1
    else
      common::runstep release::gcs::push_release_artifacts \
       $BUILD_OUTPUT-$version/gcs-stage/$version \
       gs://$RELEASE_BUCKET/$BUCKET_TYPE/$version || return 1
    fi

    common::runstep release::docker::release \
     $KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1

    common::runstep release::gcs::publish_version \
     $BUCKET_TYPE $version $BUILD_OUTPUT-$version $RELEASE_BUCKET || return 1
  fi
}

###############################################################################
# MAIN
###############################################################################
# Default mode is a mocked release workflow
: ${FLAGS_nomock:=0}
: ${FLAGS_rc:=0}
: ${FLAGS_official:=0}

BASEDIR=${FLAGS_basedir}
if [[ -z "${BASEDIR}" ]]; then
  BASEDIR="$HOME/anago"
fi

# make sure we have the compiled bins in the path
export PATH="${PATH}:${BASEDIR}/bin"

##############################################################################
# Initialize logs
##############################################################################
# Initialize and save up to 10 (rotated logs)
if ((FLAGS_stage)); then
  LOGFILE=$TMPDIR/$PROG-stage.log
else
  LOGFILE=$TMPDIR/$PROG.log
fi
common::logfileinit $LOGFILE 10
# BEGIN script
common::timestamp begin

# Point release managers to GCB as the preferred method
if ! ((FLAGS_gcb)); then
  if ((FLAGS_nomock)); then
    common::exit 1 "$FATAL: --nomock is no longer available on the" \
                   "command-line/desktop.  Use 'gcbmgr'."
  fi

  logecho $HR
  logecho "${TPUT[BOLD]}**** Welcome to the Kubernetes Release Tool, Release" \
          "Manager ****"
  logecho
  logecho "***************** $PROG has moved to the cloud! *******************${TPUT[OFF]}"
  logecho
  logecho "* Kubernetes releases have been moved off the desktop"
  logecho
  logecho "* Desktop $PROG has not been disabled, but running via GCB is the" \
          "preferred method.  Usage is similar to $PROG."
  logecho
  logecho "* https://github.com/kubernetes/release/blob/master/README.md" \
          "for details on how to get started."
  logecho $HR
  logecho
  logecho -n "Press any key to continue..."
  logrun read -n1
  logecho
fi

# Order workflow based on conditions
((FLAGS_stage)) || common::stepindex "gitlib::github_acls"
common::stepindex "check_prerequisites" "get_build_candidate" \
 "prepare_workspace" "common::disk_space_check"
common::stepindex "prepare_tree"
((FLAGS_gcb)) && common::stepindex "local_kube_cross"
if ((FLAGS_buildonly)); then
  ((FLAGS_gcb)) && common::stepindex "make_cross"
  ((FLAGS_gcb)) || common::stepindex "build_tree"
elif ! ((FLAGS_prebuild)); then
  ((FLAGS_gcb)) && common::stepindex "make_cross"
  common::stepindex "build_tree" "generate_release_notes"
  if ((FLAGS_stage)); then
    common::stepindex "stage_source_tree"
  else
    common::stepindex "push_git_objects"
  fi
  common::stepindex "push_all_artifacts"
  if ! ((FLAGS_stage)); then
    common::stepindex "announce"
    common::stepindex "update_github_release"
  fi
fi

# Show the workflow order and completed steps
common::stepheader "WORKFLOW STEPS"
common::stepindex --toc $PROGSTATE
logecho

# Pre-checks
if ! ((FLAGS_gcb)); then
  # Check tool repo
  gitlib::repo_state || common::exit 1

  # Additional functionality
  common::security_layer
fi

# Set cloud binaries
if ! ((FLAGS_buildonly)) && ! common::set_cloud_binaries; then
  logecho "Releasing Kubernetes requires gsutil and gcloud. Please download,"
  logecho "install and authorize through the Google Cloud SDK:"
  logecho
  logecho "https://developers.google.com/cloud/sdk/"
  common::exit 1 "Exiting..."
fi

# Set the majorify of global values
# Moved here b/c now depends on gcloud
release::set_globals

((FLAGS_stage)) || common::run_stateful gitlib::github_acls

# Simple check to validate who can do actual releases
if ((FLAGS_nomock)) && ! ((FLAGS_gcb)); then
  security_layer::acl_check || common::exit 1 "Exiting..."
fi

common::run_stateful "check_prerequisites"

# Call run_stateful and store these globals in the $PROGSTATE
common::run_stateful get_build_candidate JENKINS_BUILD_VERSION \
                                         PARENT_BRANCH BRANCH_POINT
# Computes a few things and creates a globals and dictionaries - not stateful
# Always Computed from JENKINS_BUILD_VERSION
set_release_values || common::exit 1 "Exiting..."

# Set values based on derived/computed values above
# WORK/BUILD area
# For --stage, it's JENKINS_BUILD_VERSION-based
if ((FLAGS_stage)); then
  WORKDIR=$BASEDIR/$PROG-$JENKINS_BUILD_VERSION
else
  WORKDIR=$BASEDIR/$PROG-$RELEASE_VERSION_PRIME
fi

if [[ $RELEASE_VERSION_PRIME =~ ${VER_REGEX[release]} ]]; then
  CHANGELOG_FILE="CHANGELOG-${BASH_REMATCH[1]}.${BASH_REMATCH[2]}.md"
else
  common::exit 1 "Unable to set CHANGELOG file!"
fi

# Go tools expect the kubernetes src to be under $GOPATH
export GOPATH=$WORKDIR
# TOOL_ROOT is release/
# TREE_ROOT is working branch/tree
TREE_ROOT=$WORKDIR/src/k8s.io/kubernetes
# Same as Makefile variable name
OUT_DIR="_output"
BUILD_OUTPUT=$TREE_ROOT/$OUT_DIR
RELEASE_NOTES_MD=$WORKDIR/src/release-notes.md
RELEASE_NOTES_HTML=$WORKDIR/src/release-notes.html

# Ensure the WORKDIR exists
logrun mkdir -p $WORKDIR

# Display top pending PRs for branch releases (convenience only)
if [[ $RELEASE_BRANCH =~ release- ]] &&
   gitlib::branch_exists $RELEASE_BRANCH; then
  common::stepheader "PENDING PRS ON THE $RELEASE_BRANCH BRANCH"
  gitlib::pending_prs $RELEASE_BRANCH
fi

# No need to pre-check this for mock or staging runs.  Overwriting OK.
if ((FLAGS_nomock)) && ! ((FLAGS_stage)); then
  common::stepheader "GCS TARGET CHECK"
  # Ensure GCS destinations are clear before continuing
  for v in ${RELEASE_VERSION[@]}; do
    release::gcs::destination_empty \
     gs://$RELEASE_BUCKET/$BUCKET_TYPE/$v || common::exit 1 "Exiting..."
  done
fi

common::stepheader "SESSION VALUES"
# Show versions and ask for confirmation to continue
# Pass in the indexed RELEASE_VERSION dict key by key
ALL_RELEASE_VERSIONS=($(for key in ${!RELEASE_VERSION[@]}; do
                         echo RELEASE_VERSION[$key]; done))

# Depending on the type of operation being performed one of these will be set
if [[ -n $BRANCH_POINT ]]; then
  DISPLAY_VERSION="BRANCH_POINT"
else
  DISPLAY_VERSION="JENKINS_BUILD_VERSION"
fi
[[ -n $PARENT_BRANCH ]] && DISPLAY_PARENT_BRANCH="PARENT_BRANCH"

common::printvars -p WORKDIR WORKDIR TREE_ROOT $DISPLAY_PARENT_BRANCH \
                     $DISPLAY_VERSION \
                     RELEASE_VERSION_PRIME ${ALL_RELEASE_VERSIONS[@]} \
                     RELEASE_BRANCH GCRIO_PATH RELEASE_BUCKET BUCKET_TYPE \
                     CHANGELOG_FILE \
                     FLAGS_nomock FLAGS_rc FLAGS_official \
                     LOGFILE

if [[ -n "$PARENT_BRANCH" ]]; then
  logecho
  logecho "$ATTENTION: $RELEASE_BRANCH is a NEW branch off $PARENT_BRANCH!"
fi

if ! ((FLAGS_nomock)); then
  logecho
  logecho "$ATTENTION: This is a mock (--mock) run." \
          "Publishing will be based on the above values for" \
          "RELEASE_BUCKET, BUCKET_TYPE and GCRIO_PATH."
fi

logecho
((FLAGS_yes)) || common::askyorn -e "Do these values look ok for a release" \
 || common::exit 1 "Exiting..."

logecho
logecho -r "${TPUT[BOLD]}>>>>>>>>${TPUT[OFF]}" \
           "View detailed session output with:  tailf $LOGFILE"
logecho -r "${TPUT[BOLD]}>>>>>>>>${TPUT[OFF]}" \
           "(Previous logs can be found in $LOGFILE.{1..10})"

common::run_stateful prepare_workspace

# Store RELEASE_GB key=value in $PROGSTATE
common::run_stateful --strip-args \
 "common::disk_space_check $BASEDIR $(($RELEASE_GB*${#RELEASE_VERSION[*]}))" \
 RELEASE_GB

# Everything happens in the TREE_ROOT context
logrun cd $TREE_ROOT

if ! ((FLAGS_stage)); then
  # Check or store STAGED_LOCATION and STAGED_BUCKET global bools
  # to ensure re-entrant non-stage runs cache this important state
  if ! common::check_state STAGED_LOCATION; then
     common::check_state -a STAGED_LOCATION STAGED_LOCATION=$STAGED_LOCATION
  fi
  if ! common::check_state STAGED_BUCKET ; then
    common::check_state -a STAGED_BUCKET STAGED_BUCKET=$STAGED_BUCKET
  fi
fi

if [[ -z $STAGED_LOCATION ]]; then
  # Iterate over session release versions for setup, tagging and building
  for label in ${!RELEASE_VERSION[@]}; do
    common::run_stateful "prepare_tree $label" RELEASE_VERSION[$label]
    # This doesn't have to be done per label, but does need to be inserted
    # after an initial prepare_tree(), so just let the statefulness of it
    # ignore a second iteration/call.
    ((FLAGS_gcb)) \
     && common::run_stateful local_kube_cross
    # --prebuild for GCB, skip actual builds. Do everything else
    if ! ((FLAGS_prebuild)); then
      if ((FLAGS_gcb)); then
        # Do only make cross in this case
        common::run_stateful "make_cross $label" RELEASE_VERSION[$label]
      else
        # Do the full build in this case
        common::run_stateful "build_tree $label" RELEASE_VERSION[$label]
      fi
    fi
  done

  # Stop here
  if ((FLAGS_prebuild)) || ((FLAGS_buildonly)); then
    logecho
    logecho "--prebuild/--buildonly complete"
    logecho
    logecho "Finish this session with:"
    logecho
    logecho "$PROG $(sed -n 's,^CMDLINE: ,,p' $PROGSTATE)"
    common::exit 0 "Exiting..."
  fi

  # Complete the "build" for the gcb case
  if ((FLAGS_gcb)); then
    if ((FLAGS_stage)); then
      for label in ${!RELEASE_VERSION[@]}; do
        common::run_stateful "build_tree $label" RELEASE_VERSION[$label]
      done
    fi
  fi

  # No release notes for X.Y.Z-beta.0 releases
  [[ -z "$PARENT_BRANCH" ]] && common::run_stateful generate_release_notes
else
  logecho
  # Force complete for these three stages
  for label in ${!RELEASE_VERSION[@]}; do
    SKIP_STEPS+=(prepare_tree+$label build_tree+$label)
    ((FLAGS_gcb)) && SKIP_STEPS+=(local_kube_cross make_cross+$label)
  done
  SKIP_STEPS+=(generate_release_notes)
  for entry in ${SKIP_STEPS[*]}; do
     # Check and add for re-entrancy
     if ! common::check_state $entry; then
       logecho "$ATTENTION: Skipping $entry step executed during staging"
       common::check_state -a $entry
     fi
  done
fi

if ((FLAGS_stage)); then
  common::run_stateful stage_source_tree
else
  common::run_stateful push_git_objects
fi

# Push for each release version of this session
for label in ${!RELEASE_VERSION[@]}; do
  common::run_stateful "push_all_artifacts $label" RELEASE_VERSION[$label]
done

# if --stage, we're done
if ((FLAGS_stage)); then
  ((FLAGS_nomock)) && EXTRA_FLAGS+=("--nomock")
  ((FLAGS_official)) && EXTRA_FLAGS+=("--official")
  ((FLAGS_rc)) && EXTRA_FLAGS+=("--rc")
  logecho
  logecho "To release this staged build, run:"
  logecho
  logecho -n "$ gcbmgr release ${EXTRA_FLAGS[*]} $RELEASE_BRANCH" \
             "--buildversion=$JENKINS_BUILD_VERSION"
  logecho
  logecho
  logecho "-OR-"
  logecho
  logecho -n "$ anago ${EXTRA_FLAGS[*]} $RELEASE_BRANCH" \
             "--buildversion=$JENKINS_BUILD_VERSION"
  logecho
  logecho

  # Move PROGSTATE
  logecho -n "Moving $PROGSTATE to $PROGSTATE.last: "
  logrun -s mv $PROGSTATE $PROGSTATE.last

  # IF GCB, keep only the last staged build
  # clean up here on success
  # Debatable if this should only happen on GCB
  if ((FLAGS_gcb)); then
    # Delete everything that's not this build in the vX.Y. namespace
    # || true to catch non-zero exit when list is empty
    if [[ $JENKINS_BUILD_VERSION =~ (v[0-9]+\.[0-9]+\.) ]]; then
      logecho "Cleaning up old staged builds from" \
              "gs://$RELEASE_BUCKET/$BUCKET_TYPE/${BASH_REMATCH[1]}..."
      $GSUTIL ls -d gs://$RELEASE_BUCKET/$BUCKET_TYPE/${BASH_REMATCH[1]}* |\
       grep -v $JENKINS_BUILD_VERSION |xargs $GSUTIL -mq rm -r || true
    fi
  fi

  common::exit 0
fi

if [[ -n "$PARENT_BRANCH" ]]; then
  # TODO: This needs the gcb create/update release issue treatment
  common::run_stateful "announce --branch"
else
  common::run_stateful announce
  common::run_stateful update_github_release
  common::stepheader "ARCHIVE RELEASE ON GS"
  common::runstep archive_release
fi

# Move PROGSTATE
logecho -n "Moving $PROGSTATE to $PROGSTATE.last: "
logrun -s mv $PROGSTATE $PROGSTATE.last

# END script
common::exit 0
